
# Create a temporary directory to hold notebook output, and change the working directory to that directory.
from tempfile import TemporaryDirectory
import os
import shutil
data_dir = TemporaryDirectory()

os.chdir(data_dir.name)


import os
if 'CRDS_CACHE_TYPE' in os.environ:
    if os.environ['CRDS_CACHE_TYPE'] == 'local':
        os.environ['CRDS_PATH'] = os.path.join(os.environ['HOME'], 'crds', 'cache')
    elif os.path.isdir(os.environ['CRDS_CACHE_TYPE']):
        os.environ['CRDS_PATH'] = os.environ['CRDS_CACHE_TYPE']
print('CRDS cache location: {}'.format(os.environ['CRDS_PATH']))

CRDS cache location: /grp/crds/cache


import warnings
import psutil
from astropy.io import fits

# Only print a DeprecationWarning the first time it shows up, not every time.
with warnings.catch_warnings():
    warnings.simplefilter("once", category=DeprecationWarning)
    import jwst
    from jwst.pipeline.calwebb_detector1 import Detector1Pipeline
    from jwst.assign_wcs.assign_wcs_step import AssignWcsStep

# The latest version of NPTT is installed in the requirements text file at:
# /jwst_validation_notebooks/environment.yml

# import NPTT
import nirspec_pipe_testing_tool as nptt

# To get data from Artifactory
from ci_watson.artifactory_helpers import get_bigdata


# Make sure that the version used is the right one

pipeline_version = jwst.__version__
nptt_version = nptt.__version__

print("Using jwst pipeline version: ", pipeline_version)
print("Using NPTT version: ", nptt_version)

Using jwst pipeline version:  1.8.2
Using NPTT version:  2.0.1


testing_data = {'fs_allslits_g140h_f100lp':{
                                  'uncal_file_nrs1': 'fs_allslits_g140h_f100lp_nrs1_uncal.fits',
                                  'uncal_file_nrs2': 'fs_allslits_g140h_f100lp_nrs2_uncal.fits',
                                  'truth_file_nrs1': 'fs_allslits_g140h_f100lp_nrs1_assign_wcs_truth.fits',
                                  'truth_file_nrs2': 'fs_allslits_g140h_f100lp_nrs2_assign_wcs_truth.fits',
                                  'msa_shutter_config': None }
               }


# define function to pull data from Artifactory
def get_artifactory_file(data_set_dict, detector):
    """This function creates a list with all the files needed per detector to run the test.
    Args:
        data_set_dict: dictionary, contains inputs for a specific mode and configuration
        detector: string, either nrs1 or nrs2
    Returns:
        data: list, contains all files needed to run test
    """
    files2obtain = ['uncal_file_nrs1', 'truth_file_nrs1', 'msa_shutter_config']
    data = []
    for file in files2obtain:
        data_file = None
        try: 
            if '_nrs' in file and '2' in detector:
                file = file.replace('_nrs1', '_nrs2')

            data_file = get_bigdata('jwst_validation_notebooks',
                                         'validation_data',
                                         'nirspec_data', 
                                         data_set_dict[file])
        except TypeError:
            data.append(None)
            continue

        data.append(data_file)

    return data


# Set common NPTT switches for this test and run the test for both detectors for each data set

# define benchmark (or 'truth') file
compare_assign_wcs_and_extract_2d_with_esa = False

# accepted threshold difference with respect to benchmark files
threshold_diff = 1e-7

# other variables
esa_files_path, raw_data_root_file = None, None
save_figs = False
show_figs = True
detectors = ['nrs1', 'nrs2']
results_dict = {}


# Test the FS data for ALLSLITS
for mode_config, data_set_dict in testing_data.items():
    if 'allslits' not in mode_config:
        continue
        
    print('Fixed slit mode:  ', mode_config)
    for det in detectors:
        print('Testing files for detector: ', det)
        data = get_artifactory_file(data_set_dict, det)
        uncal_file, truth_file, msa_shutter_config = data
        uncal_basename = os.path.basename(uncal_file)
        print('Working with uncal_file: ', uncal_basename)
        
        # Make sure that there is an assign_wcs truth product to compare to, else skip this data set
        if truth_file is None:
            print('No truth file to compare to for this detector, skipping this data set.  \n')
            skip_file = True

        # Make sure these keywords are properly set
        filt = fits.getval(uncal_file, 'FILTER')
        if 'OPAQUE' in filt or 'allslits' in uncal_basename.lower():
            if 'clear' in uncal_basename.lower():
                filt = 'CLEAR'
            else:
                l = uncal_basename.split("_")
                for li in l:
                    if 'lp' in li:
                        filt = li.upper()
                        break
            fits.setval(uncal_file, 'FILTER', value=filt)
        print('Filter = ', filt)
        if 'bots' in uncal_basename.lower():
            fits.setval(uncal_file, 'TSOVISIT', value=True)
            fits.setval(uncal_file, 'FXD_SLIT', value='S1600A1')
        elif 'fs' in uncal_basename.lower():
            fits.setval(uncal_file, 'FXD_SLIT', value='S200A1')

        # Run the stage 1 pipeline 
        print('Running detector1 pipeline...')
        rate_object = Detector1Pipeline.call(uncal_file)
        
        # Run the stage 2 pipeline steps
        print('Running spec2 pipeline...')
        try:
            assign_wcs_object = AssignWcsStep.call(rate_object)
            skip_file = False
        except:
            print("No open slits fall on detector ", det)
            print("Skipping test for this file. \n")
            skip_file = True
            
        if not skip_file:  
            # Run the validation test
            %matplotlib inline
            print('Running test for FS...')
            result, _ = nptt.calwebb_spec2_pytests.auxiliary_code.compare_wcs_fs.compare_wcs(
                                                                            assign_wcs_object, 
                                                                            truth_file=truth_file, 
                                                                            esa_files_path=esa_files_path, 
                                                                            show_figs=show_figs,
                                                                            save_figs=save_figs, 
                                                                            threshold_diff=threshold_diff, 
                                                                            raw_data_root_file=raw_data_root_file,
                                                                            output_directory=None)     
        else:
            result = 'skipped'
        
        # Did the test passed 
        print("Did FS ALLSLITS assign_wcs validation test passed? ", result, "\n\n")
        rd = {uncal_basename: result}
        results_dict.update(rd)

Fixed slit mode:   fs_allslits_g140h_f100lp
Testing files for detector:  nrs1
Working with uncal_file:  fs_allslits_g140h_f100lp_nrs1_uncal.fits
Filter =  F100LP
Running detector1 pipeline...

2022-12-01 17:33:14,291 - stpipe - INFO - PARS-DETECTOR1PIPELINE parameters found: /grp/crds/cache/references/jwst/jwst_nirspec_pars-detector1pipeline_0004.asdf
2022-12-01 17:33:14,326 - stpipe.Detector1Pipeline - INFO - Detector1Pipeline instance created.
2022-12-01 17:33:14,327 - stpipe.Detector1Pipeline.group_scale - INFO - GroupScaleStep instance created.
2022-12-01 17:33:14,328 - stpipe.Detector1Pipeline.dq_init - INFO - DQInitStep instance created.
2022-12-01 17:33:14,329 - stpipe.Detector1Pipeline.saturation - INFO - SaturationStep instance created.
2022-12-01 17:33:14,330 - stpipe.Detector1Pipeline.ipc - INFO - IPCStep instance created.
2022-12-01 17:33:14,331 - stpipe.Detector1Pipeline.superbias - INFO - SuperBiasStep instance created.
2022-12-01 17:33:14,332 - stpipe.Detector1Pipeline.refpix - INFO - RefPixStep instance created.
2022-12-01 17:33:14,333 - stpipe.Detector1Pipeline.rscd - INFO - RscdStep instance created.
2022-12-01 17:33:14,334 - stpipe.Detector1Pipeline.firstframe - INFO - FirstFrameStep instance created.
2022-12-01 17:33:14,335 - stpipe.Detector1Pipeline.lastframe - INFO - LastFrameStep instance created.
2022-12-01 17:33:14,336 - stpipe.Detector1Pipeline.linearity - INFO - LinearityStep instance created.
2022-12-01 17:33:14,337 - stpipe.Detector1Pipeline.dark_current - INFO - DarkCurrentStep instance created.
2022-12-01 17:33:14,338 - stpipe.Detector1Pipeline.reset - INFO - ResetStep instance created.
2022-12-01 17:33:14,339 - stpipe.Detector1Pipeline.persistence - INFO - PersistenceStep instance created.
2022-12-01 17:33:14,340 - stpipe.Detector1Pipeline.jump - INFO - JumpStep instance created.
2022-12-01 17:33:14,341 - stpipe.Detector1Pipeline.ramp_fit - INFO - RampFitStep instance created.
2022-12-01 17:33:14,343 - stpipe.Detector1Pipeline.gain_scale - INFO - GainScaleStep instance created.
2022-12-01 17:33:14,442 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline running with args ('/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62/fs_allslits_g140h_f100lp_nrs1_uncal.fits',).
2022-12-01 17:33:14,451 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_calibrated_ramp': False, 'steps': {'group_scale': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'dq_init': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'saturation': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'n_pix_grow_sat': 1}, 'ipc': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'superbias': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'refpix': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'odd_even_columns': True, 'use_side_ref_pixels': True, 'side_smoothing_length': 11, 'side_gain': 1.0, 'odd_even_rows': True}, 'rscd': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'type': 'baseline'}, 'firstframe': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'lastframe': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'linearity': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'dark_current': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'dark_output': None}, 'reset': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'persistence': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'input_trapsfilled': '', 'flag_pers_cutoff': 40.0, 'save_persistence': False, 'save_trapsfilled': True}, 'jump': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'rejection_threshold': 4.0, 'three_group_rejection_threshold': 6.0, 'four_group_rejection_threshold': 5.0, 'maximum_cores': 'none', 'flag_4_neighbors': True, 'max_jump_to_flag_neighbors': 1000.0, 'min_jump_to_flag_neighbors': 10.0, 'after_jump_flag_dn1': 0.0, 'after_jump_flag_time1': 0.0, 'after_jump_flag_dn2': 0.0, 'after_jump_flag_time2': 0.0, 'min_sat_area': 1.0, 'min_jump_area': 5.0, 'expand_factor': 2.0, 'use_ellipses': False, 'sat_required_snowball': True, 'expand_large_events': False}, 'ramp_fit': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'int_name': '', 'save_opt': False, 'opt_name': '', 'suppress_one_group': True, 'maximum_cores': 'none'}, 'gain_scale': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}}}
2022-12-01 17:33:14,803 - stpipe.Detector1Pipeline - INFO - Prefetching reference files for dataset: 'fs_allslits_g140h_f100lp_nrs1_uncal.fits' reftypes = ['dark', 'gain', 'linearity', 'mask', 'persat', 'readnoise', 'refpix', 'reset', 'rscd', 'saturation', 'superbias', 'trapdensity', 'trappars']
2022-12-01 17:33:14,817 - stpipe.Detector1Pipeline - INFO - Prefetch for DARK reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_dark_0078.fits'.
2022-12-01 17:33:14,819 - stpipe.Detector1Pipeline - INFO - Prefetch for GAIN reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits'.
2022-12-01 17:33:14,820 - stpipe.Detector1Pipeline - INFO - Prefetch for LINEARITY reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_linearity_0008.fits'.
2022-12-01 17:33:14,821 - stpipe.Detector1Pipeline - INFO - Prefetch for MASK reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_mask_0024.fits'.
2022-12-01 17:33:14,823 - stpipe.Detector1Pipeline - INFO - Prefetch for PERSAT reference file is 'N/A'.
2022-12-01 17:33:14,824 - stpipe.Detector1Pipeline - INFO - Prefetch for READNOISE reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0006.fits'.
2022-12-01 17:33:14,825 - stpipe.Detector1Pipeline - INFO - Prefetch for REFPIX reference file is 'N/A'.
2022-12-01 17:33:14,825 - stpipe.Detector1Pipeline - INFO - Prefetch for RESET reference file is 'N/A'.
2022-12-01 17:33:14,826 - stpipe.Detector1Pipeline - INFO - Prefetch for RSCD reference file is 'N/A'.
2022-12-01 17:33:14,826 - stpipe.Detector1Pipeline - INFO - Prefetch for SATURATION reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_saturation_0010.fits'.
2022-12-01 17:33:14,828 - stpipe.Detector1Pipeline - INFO - Prefetch for SUPERBIAS reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_superbias_0109.fits'.
2022-12-01 17:33:14,829 - stpipe.Detector1Pipeline - INFO - Prefetch for TRAPDENSITY reference file is 'N/A'.
2022-12-01 17:33:14,829 - stpipe.Detector1Pipeline - INFO - Prefetch for TRAPPARS reference file is 'N/A'.
2022-12-01 17:33:14,830 - stpipe.Detector1Pipeline - INFO - Starting calwebb_detector1 ...
2022-12-01 17:33:15,684 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:15,685 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62'}
2022-12-01 17:33:16,240 - stpipe.Detector1Pipeline.group_scale - INFO - NFRAMES and FRMDIVSR are equal; correction not needed
2022-12-01 17:33:16,242 - stpipe.Detector1Pipeline.group_scale - INFO - Step will be skipped
2022-12-01 17:33:16,244 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale done
2022-12-01 17:33:16,358 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:16,360 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62'}
2022-12-01 17:33:16,382 - stpipe.Detector1Pipeline.dq_init - INFO - Using MASK reference file /grp/crds/cache/references/jwst/jwst_nirspec_mask_0024.fits
2022-12-01 17:33:17,411 - stpipe.Detector1Pipeline.dq_init - INFO - Extracting mask subarray to match science data
2022-12-01 17:33:17,428 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init done
2022-12-01 17:33:17,532 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:17,533 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62', 'n_pix_grow_sat': 1}
2022-12-01 17:33:17,555 - stpipe.Detector1Pipeline.saturation - INFO - Using SATURATION reference file /grp/crds/cache/references/jwst/jwst_nirspec_saturation_0010.fits
2022-12-01 17:33:18,287 - stpipe.Detector1Pipeline.saturation - INFO - Extracting reference file subarray to match science data
2022-12-01 17:33:22,403 - stpipe.Detector1Pipeline.saturation - INFO - Detected 4277 saturated pixels
2022-12-01 17:33:22,473 - stpipe.Detector1Pipeline.saturation - INFO - Detected 23 A/D floor pixels
2022-12-01 17:33:22,481 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation done
2022-12-01 17:33:22,586 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:22,588 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62'}
2022-12-01 17:33:22,588 - stpipe.Detector1Pipeline.ipc - INFO - Step skipped.
2022-12-01 17:33:22,591 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc done
2022-12-01 17:33:22,687 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:22,689 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62'}
2022-12-01 17:33:22,712 - stpipe.Detector1Pipeline.superbias - INFO - Using SUPERBIAS reference file /grp/crds/cache/references/jwst/jwst_nirspec_superbias_0109.fits
2022-12-01 17:33:23,485 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias done
2022-12-01 17:33:23,614 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:23,616 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62', 'odd_even_columns': True, 'use_side_ref_pixels': True, 'side_smoothing_length': 11, 'side_gain': 1.0, 'odd_even_rows': True}
2022-12-01 17:33:24,098 - stpipe.Detector1Pipeline.refpix - INFO - NIR subarray data
2022-12-01 17:33:24,104 - stpipe.Detector1Pipeline.refpix - INFO - Single readout amplifier used
2022-12-01 17:33:24,105 - stpipe.Detector1Pipeline.refpix - INFO - The following parameter is valid for this mode:
2022-12-01 17:33:24,105 - stpipe.Detector1Pipeline.refpix - INFO - odd_even_columns = True
2022-12-01 17:33:24,105 - stpipe.Detector1Pipeline.refpix - INFO - The following parameters are not applicable and are ignored:
2022-12-01 17:33:24,106 - stpipe.Detector1Pipeline.refpix - INFO - use_side_ref_pixels = True
2022-12-01 17:33:24,106 - stpipe.Detector1Pipeline.refpix - INFO - side_smoothing_length = 11
2022-12-01 17:33:24,106 - stpipe.Detector1Pipeline.refpix - INFO - side_gain = 1.0
2022-12-01 17:33:24,106 - stpipe.Detector1Pipeline.refpix - INFO - odd_even_rows = False
2022-12-01 17:33:24,142 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix done
2022-12-01 17:33:24,251 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:24,252 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62'}
2022-12-01 17:33:24,275 - stpipe.Detector1Pipeline.linearity - INFO - Using Linearity reference file /grp/crds/cache/references/jwst/jwst_nirspec_linearity_0008.fits
2022-12-01 17:33:28,128 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity done
2022-12-01 17:33:28,247 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:28,249 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62', 'dark_output': None}
2022-12-01 17:33:28,272 - stpipe.Detector1Pipeline.dark_current - INFO - Using DARK reference file /grp/crds/cache/references/jwst/jwst_nirspec_dark_0078.fits
2022-12-01 17:33:29,983 - stpipe.Detector1Pipeline.dark_current - INFO - Science data nints=25, ngroups=10, nframes=1, groupgap=0
2022-12-01 17:33:29,984 - stpipe.Detector1Pipeline.dark_current - INFO - Dark data nints=1, ngroups=10, nframes=1, groupgap=0
2022-12-01 17:33:31,232 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current done
2022-12-01 17:33:31,458 - stpipe.Detector1Pipeline.jump - INFO - Step jump running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 17:33:31,460 - stpipe.Detector1Pipeline.jump - INFO - Step jump parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62', 'rejection_threshold': 4.0, 'three_group_rejection_threshold': 6.0, 'four_group_rejection_threshold': 5.0, 'maximum_cores': 'none', 'flag_4_neighbors': True, 'max_jump_to_flag_neighbors': 1000.0, 'min_jump_to_flag_neighbors': 10.0, 'after_jump_flag_dn1': 0.0, 'after_jump_flag_time1': 0.0, 'after_jump_flag_dn2': 0.0, 'after_jump_flag_time2': 0.0, 'min_sat_area': 1.0, 'min_jump_area': 5.0, 'expand_factor': 2.0, 'use_ellipses': False, 'sat_required_snowball': True, 'expand_large_events': False}
2022-12-01 17:33:31,473 - stpipe.Detector1Pipeline.jump - INFO - CR rejection threshold = 4 sigma
2022-12-01 17:33:31,489 - stpipe.Detector1Pipeline.jump - INFO - Using GAIN reference file: /grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits
2022-12-01 17:33:31,776 - stpipe.Detector1Pipeline.jump - INFO - Using READNOISE reference file: /grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0006.fits
2022-12-01 17:33:32,517 - stpipe.Detector1Pipeline.jump - INFO - Extracting gain subarray to match science data
2022-12-01 17:33:32,518 - stpipe.Detector1Pipeline.jump - INFO - Extracting readnoise subarray to match science data
2022-12-01 17:33:32,827 - stpipe.Detector1Pipeline.jump - INFO - Executing two-point difference method
2022-12-01 17:33:33,146 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 1:
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/twopoint_difference.py:154: RuntimeWarning: All-NaN slice encountered
  max_ratio = np.nanmax(ratio, axis=0)
2022-12-01 17:33:33,529 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 281332 pixels with at least one CR from five or more groups.
2022-12-01 17:34:11,411 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 2:
2022-12-01 17:34:11,785 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 501912 pixels with at least one CR from five or more groups.
2022-12-01 17:36:06,969 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 3:
2022-12-01 17:36:07,331 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 508309 pixels with at least one CR from five or more groups.
2022-12-01 17:37:17,640 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 4:
2022-12-01 17:37:18,008 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 371800 pixels with at least one CR from five or more groups.
2022-12-01 17:38:15,140 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 5:
2022-12-01 17:38:15,492 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 516917 pixels with at least one CR from five or more groups.
2022-12-01 17:40:51,966 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 6:
2022-12-01 17:40:52,335 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 203498 pixels with at least one CR from five or more groups.
2022-12-01 17:41:25,570 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 7:
2022-12-01 17:41:25,948 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 114624 pixels with at least one CR from five or more groups.
2022-12-01 17:41:41,295 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 8:
2022-12-01 17:41:41,658 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 508629 pixels with at least one CR from five or more groups.
2022-12-01 17:43:58,047 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 9:
2022-12-01 17:43:58,411 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 491934 pixels with at least one CR from five or more groups.
2022-12-01 17:45:20,964 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 10:
2022-12-01 17:45:21,332 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 496084 pixels with at least one CR from five or more groups.
2022-12-01 17:46:25,445 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 11:
2022-12-01 17:46:25,807 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 500227 pixels with at least one CR from five or more groups.
2022-12-01 17:47:38,353 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 12:
2022-12-01 17:47:38,719 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 40867 pixels with at least one CR from five or more groups.
2022-12-01 17:47:44,365 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 13:
2022-12-01 17:47:44,729 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 68365 pixels with at least one CR from five or more groups.
2022-12-01 17:47:54,350 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 14:
2022-12-01 17:47:54,714 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 123230 pixels with at least one CR from five or more groups.
2022-12-01 17:48:11,886 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 15:
2022-12-01 17:48:12,237 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 515522 pixels with at least one CR from five or more groups.
2022-12-01 17:51:12,334 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 16:
2022-12-01 17:51:12,702 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 427416 pixels with at least one CR from five or more groups.
2022-12-01 17:52:20,328 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 17:
2022-12-01 17:52:20,700 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 410843 pixels with at least one CR from five or more groups.
2022-12-01 17:53:19,031 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 18:
2022-12-01 17:53:19,394 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 361479 pixels with at least one CR from five or more groups.
2022-12-01 17:54:29,887 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 19:
2022-12-01 17:54:30,262 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 496296 pixels with at least one CR from five or more groups.
2022-12-01 17:56:24,656 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 20:
2022-12-01 17:56:25,018 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 503654 pixels with at least one CR from five or more groups.
2022-12-01 17:58:20,346 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 21:
2022-12-01 17:58:20,716 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 223426 pixels with at least one CR from five or more groups.
2022-12-01 17:58:51,372 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 22:
2022-12-01 17:58:51,735 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 512792 pixels with at least one CR from five or more groups.
2022-12-01 18:01:48,987 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 23:
2022-12-01 18:01:49,365 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 513716 pixels with at least one CR from five or more groups.
2022-12-01 18:05:01,793 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 24:
2022-12-01 18:05:02,154 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 497651 pixels with at least one CR from five or more groups.
2022-12-01 18:06:57,825 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 25:
2022-12-01 18:06:58,202 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 38554 pixels with at least one CR from five or more groups.
2022-12-01 18:07:03,825 - stpipe.Detector1Pipeline.jump - INFO - Total elapsed time = 2011 sec
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/jump.py:302: RuntimeWarning: invalid value encountered in divide
  data /= gain_2d
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/jump.py:303: RuntimeWarning: invalid value encountered in divide
  err /= gain_2d
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/jump.py:304: RuntimeWarning: invalid value encountered in divide
  readnoise_2d /= gain_2d
2022-12-01 18:07:04,239 - stpipe.Detector1Pipeline.jump - INFO - The execution time in seconds: 2012.766704
2022-12-01 18:07:04,245 - stpipe.Detector1Pipeline.jump - INFO - Step jump done
2022-12-01 18:07:04,424 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit running with args (<RampModel(25, 10, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 18:07:04,425 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62', 'int_name': '', 'save_opt': False, 'opt_name': '', 'suppress_one_group': True, 'maximum_cores': 'none'}
2022-12-01 18:07:04,690 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using READNOISE reference file: /grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0006.fits
2022-12-01 18:07:04,690 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using GAIN reference file: /grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits
2022-12-01 18:07:08,103 - stpipe.Detector1Pipeline.ramp_fit - INFO - Extracting gain subarray to match science data
2022-12-01 18:07:08,105 - stpipe.Detector1Pipeline.ramp_fit - INFO - Extracting readnoise subarray to match science data
2022-12-01 18:07:08,106 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using algorithm = ols
2022-12-01 18:07:08,107 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using weighting = optimal
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
2022-12-01 18:10:53,875 - stpipe.Detector1Pipeline.ramp_fit - INFO - Number of groups per integration: 10
2022-12-01 18:10:53,875 - stpipe.Detector1Pipeline.ramp_fit - INFO - Number of integrations: 25
2022-12-01 18:10:53,991 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit done
2022-12-01 18:10:54,137 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale running with args (<ImageModel(256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 18:10:54,139 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'gain_scale', 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62'}
2022-12-01 18:10:54,352 - stpipe.Detector1Pipeline.gain_scale - INFO - GAINFACT not found in gain reference file
2022-12-01 18:10:54,352 - stpipe.Detector1Pipeline.gain_scale - INFO - Step will be skipped
2022-12-01 18:10:54,357 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale done
2022-12-01 18:10:54,460 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale running with args (<CubeModel(25, 256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 18:10:54,462 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'gain_scaleints', 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62'}
2022-12-01 18:10:54,547 - stpipe.Detector1Pipeline.gain_scale - INFO - GAINFACT not found in gain reference file
2022-12-01 18:10:54,548 - stpipe.Detector1Pipeline.gain_scale - INFO - Step will be skipped
2022-12-01 18:10:54,552 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale done
2022-12-01 18:10:54,553 - stpipe.Detector1Pipeline - INFO - ... ending calwebb_detector1
2022-12-01 18:10:54,553 - stpipe.Detector1Pipeline - INFO - Results used CRDS context: jwst_1018.pmap
2022-12-01 18:10:54,554 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline done
2022-12-01 18:10:54,562 - stpipe.AssignWcsStep - INFO - AssignWcsStep instance created.
2022-12-01 18:10:54,666 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep running with args (<ImageModel(256, 2048) from fs_allslits_g140h_f100lp_nrs1_uncal.fits>,).
2022-12-01 18:10:54,668 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'sip_approx': True, 'sip_max_pix_error': 0.1, 'sip_degree': None, 'sip_max_inv_pix_error': 0.1, 'sip_inv_degree': None, 'sip_npoints': 12, 'slit_y_low': -0.55, 'slit_y_high': 0.55}

Running spec2 pipeline...

2022-12-01 18:10:55,156 - stpipe.AssignWcsStep - INFO - gwa_ytilt is 0.1343955546617508 deg
2022-12-01 18:10:55,157 - stpipe.AssignWcsStep - INFO - gwa_xtilt is 0.3604099452495575 deg
2022-12-01 18:10:55,158 - stpipe.AssignWcsStep - INFO - theta_y correction: 0.0019762940493390285 deg
2022-12-01 18:10:55,159 - stpipe.AssignWcsStep - INFO - theta_x correction: 0.0 deg
2022-12-01 18:10:55,533 - stpipe.AssignWcsStep - INFO - Removing slit S200B1 from the list of open slits because the WCS bounding_box is completely outside the detector.
2022-12-01 18:10:55,534 - stpipe.AssignWcsStep - INFO - Slits projected on detector NRS1: ['S200A1', 'S200A2', 'S400A1', 'S1600A1']
2022-12-01 18:10:55,535 - stpipe.AssignWcsStep - INFO - Computing WCS for 4 open slitlets
2022-12-01 18:10:55,574 - stpipe.AssignWcsStep - INFO - gwa_ytilt is 0.1343955546617508 deg
2022-12-01 18:10:55,575 - stpipe.AssignWcsStep - INFO - gwa_xtilt is 0.3604099452495575 deg
2022-12-01 18:10:55,576 - stpipe.AssignWcsStep - INFO - theta_y correction: 0.0019762940493390285 deg
2022-12-01 18:10:55,577 - stpipe.AssignWcsStep - INFO - theta_x correction: 0.0 deg
2022-12-01 18:10:55,592 - stpipe.AssignWcsStep - INFO - SPORDER= -1, wrange=[9.7e-07, 1.89e-06]
2022-12-01 18:10:55,770 - stpipe.AssignWcsStep - INFO - There are 0 open slits in quadrant 1
2022-12-01 18:10:55,771 - stpipe.AssignWcsStep - INFO - There are 0 open slits in quadrant 2
2022-12-01 18:10:55,772 - stpipe.AssignWcsStep - INFO - There are 0 open slits in quadrant 3
2022-12-01 18:10:55,772 - stpipe.AssignWcsStep - INFO - There are 0 open slits in quadrant 4
2022-12-01 18:10:55,773 - stpipe.AssignWcsStep - INFO - There are 4 open slits in quadrant 5
2022-12-01 18:10:55,992 - stpipe.AssignWcsStep - INFO - Created a NIRSPEC nrs_fixedslit pipeline with references {'distortion': None, 'filteroffset': None, 'specwcs': None, 'regions': None, 'wavelengthrange': '/grp/crds/cache/references/jwst/jwst_nirspec_wavelengthrange_0004.asdf', 'camera': '/grp/crds/cache/references/jwst/jwst_nirspec_camera_0004.asdf', 'collimator': '/grp/crds/cache/references/jwst/jwst_nirspec_collimator_0004.asdf', 'disperser': '/grp/crds/cache/references/jwst/jwst_nirspec_disperser_0033.asdf', 'fore': '/grp/crds/cache/references/jwst/jwst_nirspec_fore_0023.asdf', 'fpa': '/grp/crds/cache/references/jwst/jwst_nirspec_fpa_0005.asdf', 'msa': '/grp/crds/cache/references/jwst/jwst_nirspec_msa_0005.asdf', 'ote': '/grp/crds/cache/references/jwst/jwst_nirspec_ote_0005.asdf', 'ifupost': None, 'ifufore': None, 'ifuslicer': None}
2022-12-01 18:10:56,322 - stpipe.AssignWcsStep - INFO - COMPLETED assign_wcs
2022-12-01 18:10:56,335 - stpipe.AssignWcsStep - INFO - Results used CRDS context: jwst_1018.pmap
2022-12-01 18:10:56,336 - stpipe.AssignWcsStep - INFO - Step AssignWcsStep done

Running test for FS...
Information from the 'truth' (or comparison) file 
Filename: /internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62/fs_allslits_g140h_f100lp_nrs1_assign_wcs_truth.fits
No.    Name      Ver    Type      Cards   Dimensions   Format
  0  PRIMARY       1 PrimaryHDU     264   ()      
  1  SCI           1 ImageHDU        50   (2048, 256)   float32   
  2  ERR           1 ImageHDU        10   (2048, 256)   float32   
  3  DQ            1 ImageHDU        11   (2048, 256)   int32 (rescales to uint32)   
  4  VAR_POISSON    1 ImageHDU         9   (2048, 256)   float32   
  5  VAR_RNOISE    1 ImageHDU         9   (2048, 256)   float32   
  6  ASDF          1 BinTableHDU     11   1R x 1C   [191182B]   
None
Comparing to ST 'truth' file.
from assign_wcs file  -->     Detector: NRS1   Grating: G140H   Filter: F100LP   Lamp: NO_LAMP
GWA_XTILT: 0.3604099452495575
GWA_YTILT: 0.1343955546617508
GWA_TTILT: 37.0910277792677

Working with slit: S200A1
Avalable frames:  ['detector', 'sca', 'gwa', 'slit_frame', 'msa_frame', 'oteip', 'v2v3', 'v2v3vacorr', 'world']

   Relative Wavelength Difference  :   mean = nan    median = nan    stdev = nan
    Maximum   RelativeWavelength Difference = nan
    Minimum   RelativeWavelength Difference = nan
    Percentage of pixels where median of relative differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%

 * Result of the test for Wavelength Difference:  FAILED


   Relative Slit-Y Difference  :   mean = 0.000e+00    median = 0.000e+00    stdev = 0.000e+00
    Maximum   RelativeSlit-Y Difference = -0.000e+00
    Minimum   RelativeSlit-Y Difference = -0.000e+00
    Percentage of pixels where median of relative differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%

 * Result of the test for Slit-Y Difference:  PASSED


   Relative MSA_X Difference  :   mean = 0.000e+00    median = 0.000e+00    stdev = 0.000e+00
    Maximum   RelativeMSA_X Difference = 0.000e+00
    Minimum   RelativeMSA_X Difference = 0.000e+00
    Percentage of pixels where median of relative differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%

 * Result of the test for MSA_X Difference:  PASSED


   Relative MSA_Y Difference  :   mean = 0.000e+00    median = 0.000e+00    stdev = 0.000e+00
    Maximum   RelativeMSA_Y Difference = -0.000e+00
    Minimum   RelativeMSA_Y Difference = -0.000e+00
    Percentage of pixels where median of relative differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%

 * Result of the test for MSA_Y Difference:  PASSED


   Relative V2 difference  :   mean = 0.000e+00    median = 0.000e+00    stdev = 0.000e+00
    Maximum   RelativeV2 difference = 0.000e+00
    Minimum   RelativeV2 difference = 0.000e+00
    Percentage of pixels where median of relative differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%

 * Result of the test for V2 difference:  PASSED


   Relative V3 difference  :   mean = 0.000e+00    median = 0.000e+00    stdev = 0.000e+00
    Maximum   RelativeV3 difference = -0.000e+00
    Minimum   RelativeV3 difference = -0.000e+00
    Percentage of pixels where median of relative differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%

 * Result of the test for V3 difference:  PASSED

No output_directory was provided. Figures will be saved in current working directory:
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp0emcad62/S200A1_NRS1_rel_wave_diffs.png

No output_directory was provided. Figures will NOT be saved.

No output_directory was provided. Figures will NOT be saved.

No output_directory was provided. Figures will NOT be saved.

No output_directory was provided. Figures will NOT be saved.

No output_directory was provided. Figures will NOT be saved.


# Summary of modes tested and their results 

print('These are the final results of the tests: ')
for key, val in results_dict.items():
    if not isinstance(val, str):
        if val:
            val = 'PASSED'
        else:
            val = 'FAILED'
    print('{:<42} {:<8}'.format(key, val))

These are the final results of the tests: 
fs_allslits_g140h_f100lp_nrs1_uncal.fits   PASSED  
fs_allslits_g140h_f100lp_nrs2_uncal.fits   FAILED

JWST Pipeline Validation Testing Notebook: NIRSpec, spec2, assign_wcs step for fs allslits¶

Table of Contents¶

Imports¶

Test Description¶

Results¶

Calibration WG Requested Algorithm:¶

Defining Term¶

Run the JWST pipeline and assign_wcs validation tests¶

Testing Data Set¶

About this Notebook¶